package com.chance.cc.crawler.development.bootstrap.bitauto;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.interaction;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Site_Info;

/**
 * @Author: ZhaoHhuan on 2021/9/3 16:45
 * @Email: 18638575967@163.com
 * @Description:
 *      易车文章放入请求队列中
**/
public class BitautoArticleUrlListToRedisStart {
    public static final String domainId = "bitauto";
    private static final List<String> urlList = Arrays.asList("https://www.pcauto.com.cn/nation/2576/25767151.html");

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    HttpConfig httpConfig = HttpConfig.me(domainId);


    public static void main(String[] args) {
        createItemRecordToRedis();
    }

    //将文件中的地址生成record放入redis中
    private static void createItemRecordToRedis(){
        String site = "video";
        String siteBiz = "article";
        String siteInfo = "article-simple-20210903";
        String path = "F:\\需求文件\\8_yichewang_shiping.json";

        HttpConfig httpConfig = HttpConfig.me(domainId);
//        httpConfig.setCircularRedirectsAllowed(true);
        //push kv first
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
//
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
//            for (String readLine : urlList) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                JSONObject jsonObject = JSONObject.parseObject(readLine);
                String url = jsonObject.getString("url");
                url = url + "#no_date";
//                String url = readLine;
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .domain(domainId)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .httpConfig(HttpConfig.me(domainId))
                        .filter(CrawlerEnum.CrawlerRecordFilter.key)
                        .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 50,null))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                        .proxy(proxy)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,siteInfo);
                Map<String,Object> extras = new HashMap<>();
                extras.put("listUrl","lastRequestUrl");
                extras.put("itemUrl",url);
                HttpRequest httpRequest = requestRecord.getHttpRequest();
                httpRequest.setExtras(extras);

                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);

                redisReader.mapPush("crawler-bitauto-article-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-bitauto-article-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
